package com.xuxueli.crawler.test;

import com.gargoylesoftware.htmlunit.Page;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.annotation.PageFieldSelect;
import com.xuxueli.crawler.annotation.PageSelect;
import com.xuxueli.crawler.conf.XxlCrawlerConf;
import com.xuxueli.crawler.parser.PageParser;
import com.xuxueli.crawler.proxy.ProxyMaker;
import com.xuxueli.crawler.proxy.strategy.RoundProxyMaker;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.ArrayList;
import java.util.List;

/**
 * 爬虫示例01：爬取页面数据并封装VO对象
 *
 * @author xuxueli 2017-10-09 19:48:48
 */
public class XxlCrawlerTest8 {
    @PageSelect(cssQuery = ".newest-soft.bluea")
    public static class PageVo {
        @PageFieldSelect(cssQuery = "li h3 a")
        public List<String> title;            //标题
        @PageFieldSelect(cssQuery = "li h3 a", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:href")
        public List<String> url;

        public List<String> getTitle() {
            return title;
        }

        public void setTitle(List<String> title) {
            this.title = title;
        }

        public List<String> getUrl() {
            return url;
        }

        public void setUrl(List<String> url) {
            this.url = url;
        }

        @Override
        public String toString() {
            return "PageVo{" +
                    "title=" + title +
                    ", url=" + url +
                    '}';
        }
    }

    //----------------------------------
    @PageSelect(cssQuery = "#mainBody")
    public static class PageTwiceVo {
        @PageFieldSelect(cssQuery = "#main h1")
        public String title;            //标题
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(0)")
        public String lang;             //语言
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(1)")
        public String size;                //包大小
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(2) span")
        public int times;//下载次数
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(3) span")
        public int views;//浏览次数
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(4) span")
        public String date;             //发布时间
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(5)")
        public String category;         //类别
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(6) a")
        public String publisher;        //发布人
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(6) a", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:href")
        public String publisherUrl;        //发布人的主页
        @PageFieldSelect(cssQuery = ".param-ul.bluea li:eq(7)")
        public String Score;               //所需积分
        @PageFieldSelect(cssQuery = ".lan.zwch a")
        public List<String> tag;              //相关标签 ××
        @PageFieldSelect(cssQuery = "#soft-intro", selectType = XxlCrawlerConf.SelectType.HTML)
        public String detail;           //实例介绍
        @PageFieldSelect(cssQuery = "#dingandcai a", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:href")
        public String downloadUrl;      //下载地址

        @Override
        public String toString() {
            return "PageTwiceVo{" +
                    "title='" + title + '\'' +
                    ", lang='" + lang + '\'' +
                    ", size='" + size + '\'' +
                    ", times=" + times +
                    ", views=" + views +
                    ", date='" + date + '\'' +
                    ", category='" + category + '\'' +
                    ", publisher='" + publisher + '\'' +
                    ", publisherUrl='" + publisherUrl + '\'' +
                    ", Score='" + Score + '\'' +
                    ", tag=" + tag +
                    ", detail='" + detail + '\'' +
                    ", downloadUrl='" + downloadUrl + '\'' +
                    '}';
        }

        public String getTitle() {
            return title;
        }

        public void setTitle(String title) {
            this.title = title;
        }

        public String getLang() {
            return lang;
        }

        public void setLang(String lang) {
            this.lang = lang;
        }

        public String getSize() {
            return size;
        }

        public void setSize(String size) {
            this.size = size;
        }

        public int getTimes() {
            return times;
        }

        public void setTimes(int times) {
            this.times = times;
        }

        public String getDate() {
            return date;
        }

        public void setDate(String date) {
            this.date = date;
        }

        public String getCategory() {
            return category;
        }

        public void setCategory(String category) {
            this.category = category;
        }

        public String getPublisher() {
            return publisher;
        }

        public void setPublisher(String publisher) {
            this.publisher = publisher;
        }

        public String getPublisherUrl() {
            return publisherUrl;
        }

        public void setPublisherUrl(String publisherUrl) {
            this.publisherUrl = publisherUrl;
        }

        public String getScore() {
            return Score;
        }

        public void setScore(String score) {
            Score = score;
        }

        public List<String> getTag() {
            return tag;
        }

        public void setTag(List<String> tag) {
            this.tag = tag;
        }

        public String getDetail() {
            return detail;
        }

        public void setDetail(String detail) {
            this.detail = detail;
        }

        public String getDownloadUrl() {
            return downloadUrl;
        }

        public void setDownloadUrl(String downloadUrl) {
            this.downloadUrl = downloadUrl;
        }

        public int getViews() {
            return views;
        }

        public void setViews(int views) {
            this.views = views;
        }
    }

    public static void main(String[] args) {
        // 设置代理池
        ProxyMaker proxyMaker = new RoundProxyMaker()
                .addProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("---", 80)));

        //list封装内容
        final List<String> urlList = new ArrayList<>();
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls("https://www.haolizi.net/examples/java_1.html")
                .setWhiteUrlRegexs("https://www.haolizi.net/examples/java_\\d*.html")
                .setThreadCount(3)
                .setPageParser(new PageParser<PageVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, PageVo pageVo) {
                        // 解析封装 PageVo 对象
                        String pageUrl = html.baseUri();
                        urlList.addAll(pageVo.url);
                        //将其写进数据库
                        System.out.println(Thread.currentThread().getName() + pageUrl + "：" + pageVo.toString());
                    }
                })
                .build();
        System.out.println("start");
        crawler.start(true);
        System.out.println("end");
        String[] arr = urlList.toArray(new String[urlList.size()]);
        XxlCrawler crawlerTwice = new XxlCrawler.Builder()
                .setUrls(arr)
                .setWhiteUrlRegexs(arr)
                .setThreadCount(3)
                .setPageParser(new PageParser<PageTwiceVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, PageTwiceVo pageTwiceVo) {
                        // 解析封装 PageVo 对象
                        String pageUrl = html.baseUri();
                        System.out.println(Thread.currentThread().getName() + pageUrl + "：" + pageTwiceVo.toString());
                    }
                })
                .build();
        System.out.println("=========================start  twice===================");
        crawlerTwice.start(true);
        System.out.println("end twice");
    }
}
